using System;
using System.Net;
using System.Net.Sockets;
using System.Text.RegularExpressions;
using System.Text;
using System.Threading;

using Jeffrey.XFramework.XCrawl;
using Jeffrey.XFramework.XInteract;

namespace Jeffrey.XFramework.XNet
{
    /*
     * Implement the Http-Response via socket connection
     */
    public class XWebResponse
    {
        //the buffer to receive the http-header from server...
        private static readonly Int32 s_HeaderBufferSize  = 0x4;
        //the buffer to receive the message body from server...
        private static readonly Int32 s_ContentBufferSize = 0x2000; /* size of 8K */
        //the pattern to separate the header properties' name and value
        private static Regex s_HeaderExtractor = new Regex("[^\r\n]+");

        //only can be invoked by xrequest
        internal XWebResponse(CrawlController cc) 
        {
            //for HTTP 1.1 the default value is true.
            m_KeepAlive     = true;
            //-1 means not available
            m_ContentLength = -1;

            m_Controller = cc;
            m_Encoding = CrawlOrder.PrimEncoding;
            m_RedirectedPage = String.Empty;
        }

        internal void Reset()
        {
            m_ContentLength = -1;
            m_Encoding = CrawlOrder.PrimEncoding;
            m_RedirectedPage = String.Empty;
        }

        //flaged the whether connection is allowed to alive
        public Boolean KeepAlive
        {
            get { return m_KeepAlive; }
        }
        //the current processing uri...
        public CrawlUri CurrentUri
        {
            get { return m_Uri; }
        }
        //get the content charset...
        public Encoding Encode
        {
            get { return m_Encoding; }
        }
        public String RedirectedPage
        {
            get { return m_RedirectedPage; }
        }
        /*
         * connecting to the server using xrequest via socket
         */
        public void Connect(XWebRequest xRequest)
        {
            //set the current uri to response
            m_Uri = xRequest.m_Uri;
            String Host = m_Uri.Host;

            try
            {
                //get a new socket instance
                m_Socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
                //the default value is... true;
                m_Socket.Blocking = true;
                //get the host's ip
                IPAddress[] ips;

                if (m_Controller.m_CrawlOrder.m_DnsCache)
                    ips = DnsCache.DnsLookUp(m_Uri.Host);
                else
                    ips = Dns.GetHostAddresses(m_Uri.Host);

                if (ips == null || ips.Length == 0)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_ResolveHost;
                    return;
                }

                Exception exception = null;

                foreach (IPAddress ip in ips)
                {
                    try
                    {
                        //connecting to the specified ip and port...
                        m_Socket.Connect(ip, m_Uri.Port);
                        //cache the succeeded ip to the pool...
                        DnsCache.CacheIP(Host, ip);
                        break;
                    }
                    catch (Exception ex)
                    {
                        exception = ex;
                    }
                }

                if (!m_Socket.Connected)
                {
                    DnsCache.CacheFailed(Host);
                    if (exception != null)
                        throw exception;
                }
            }
            catch (SocketException se)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_Connected;
                m_Uri.m_SocketError = se.Message;
            }
            catch (ObjectDisposedException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_Connected;
            }
            catch (ArgumentException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_Connected;
            }
        }

        //set timeout for sending and receiving
        public void SetTimeOut(Int32 TimeOut)
        {
            try
            {
                m_Socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendTimeout, TimeOut);
                m_Socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveTimeout, TimeOut);

                m_TimeOut = TimeOut;
            }
            catch (SocketException se)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_SetTimeOut;
                m_Uri.m_SocketError = se.Message;
            }
            catch (ObjectDisposedException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_SetTimeOut;
            }
        }

        //sent the request header to the specified web server
        public void Sent(XWebRequest xRequest)
        {
            //set the current uri to response
            m_Uri = xRequest.m_Uri;
            //initialize the header string
            xRequest.m_Header = "";

            /*
             * The http-request header goes like this:
             * 
             * GET /xxx.html HTTP/1.1
             * Host: www.xxx.com.cn
             */
            xRequest.m_Header += xRequest.m_Method + " " + xRequest.m_Uri.PathAndQuery + " HTTP/1.1\r\n" + xRequest.m_Headers;

            //this should always be "ISO-8859-1"...
            Byte[] buffer = CrawlOrder.PrimEncoding.GetBytes(xRequest.m_Header);

            //sent the request to web-server
            try
            {
                SendEx(buffer);
            }
            catch (SocketException se)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_RequestSend;
                m_Uri.m_SocketError = se.Message;
            }
            catch (ArgumentException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_RequestSend;
            }
            catch (ObjectDisposedException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_RequestSend;
            }
        }

        /*
         * Receive the Server's response header
         */
        public void ReceiveHTTPHeader()
        {
            //initializing...
            m_Header  = String.Empty;
            m_Headers = new WebHeaderCollection();

            Byte[] buffer = new Byte[s_HeaderBufferSize];

            /*
             * only get one byte from header each time
             * if the current byte stands for '\n'
             * and the revceived string ended with "\r\n\r\n",
             * it flags the header has been completely received.
             */
            try
            {
                while (true)
                {
                    //encoding the gotten byte and adding to the header string
                    m_Header += ReceiveEx(buffer, 0, 1, SocketFlags.None);

                    if ( m_Header.EndsWith("\r\n\r\n") ||
                         //in case some server...
                         m_Header.EndsWith("\n\n")
                       )
                        break;
                }
            }
            catch (SocketException se)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_HeaderReceive;
                m_Uri.m_SocketError = se.Message;
                return;
            }
            catch (ArgumentException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_HeaderReceive;
                return;
            }
            catch (ObjectDisposedException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_HeaderReceive;
                return;
            }
            catch (NotReceivedExpectedSizeException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_NeedRetry;
                return;
            }

            //separate the header's properties to name and value
            MatchCollection matches = s_HeaderExtractor.Matches(m_Header.TrimEnd('\r','\n'));

            if (matches.Count > 0)
            {
                /*
                 * To get the Status-Code of the response header
                 * 
                 * Status-Line = HTTP-Version SP Status-Code SP Reason-Phrase CRLF
                 */
                String[] StatusItem = matches[0].Value.Split(new char[] { ' ' });
                try
                {
                    m_Uri.m_ResponseCode = Int32.Parse(StatusItem[1]);
                }
                catch (FormatException)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_ResponsedCodeParse;
                    return;
                }

                /*
                 * check the status code
                 * if bigger than 302, we do not process any more...
                 */
                if (m_Uri.m_ResponseCode >= 400)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_ResponsedErrorCode;
                    return;
                }
            }

            try
            {
                /*
                 * To skip the first line "HTTP/1.1 xxx xxx", so start at index 1 to
                 * get the properties' name and value
                 */
                for (Int32 i = 1; i < matches.Count; i++)
                {
                    //separate the name and value from string
                    String[] HeadItem = matches[i].Value.Split(new Char[] { ':' }, 2);
                    //add them to the Headers
                    if (HeadItem.Length > 0)
                        //change all the header names to lower case...
                        m_Headers[HeadItem[0].ToLower().Trim()] = HeadItem[1].Trim();
                }
            }
            catch (Exception)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_HeaderParse;
                return;
            }
            /*
             * Processing the page redirection...
             */
            if (  matches.Count > 0 &&
                 (m_Uri.m_ResponseCode == 201 || //Created, the new document url in [Location]
                  m_Uri.m_ResponseCode == 301 ||
                  m_Uri.m_ResponseCode == 302 ||
                  m_Uri.m_ResponseCode == 303 ||
                  m_Uri.m_ResponseCode == 307)
                )
            {
                //check if the server tells the target uri
                if (m_Headers["location"] != null)
                {
                    try
                    {
                        m_RedirectedPage = new CrawlUri(m_Uri, m_Headers["location"]).AbsoluteUri;
                    }
                    catch (UriFormatException)
                    {
                        m_Uri.m_Status = CrawlUri.StatusCode.E_CannotDeterminePage;
                    }
                    catch (ArgumentException)
                    {
                        m_Uri.m_Status = CrawlUri.StatusCode.E_CannotDeterminePage;
                    }
                }
                else
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_CannotDeterminePage;
                }
            }

            //check if the server flaged the message-body length
            if (m_Headers["content-length"] != null)
                m_ContentLength = Int32.Parse(m_Headers["content-length"]);

            /*
             * check if the server allowed the connection to be kept alive...
             * if the connection header do not transforred explicitly, we assume that the connection is persistent.
             */
            m_KeepAlive = (!(m_Headers["connection"] != null && m_Headers["connection"].ToLower() == "close") ||
                            (m_Headers["proxy-connection"] != null && m_Headers["proxy-connection"].ToLower() == "close")
                           );

            //try to determine the content charset...
            if (m_Headers["content-type"] != null)
            {
                String[] subItme = m_Headers["content-type"].Split(new Char[] { ';' });

                if (subItme.Length > 0)
                {
                    String charset = null;

                    foreach (String s in subItme)
                    {
                        if (s.Trim(' ').ToLower().IndexOf("charset") != -1)
                        {
                            Int32 index = s.IndexOf("=");
                            if (index != -1)
                                charset = s.Substring(index + 1).Trim(' ');
                            break;
                        }
                    }
                    if (charset != null)
                    {
                        switch (charset.ToLower())
                        {
                            case "utf8":  charset = "UTF-8";  break;
                            case "utf16": charset = "UTF-16"; break;
                            case "utf32": charset = "UTF-32"; break;
                            default: break;
                        }
                        try
                        {
                            m_Encoding = Encoding.GetEncoding(charset);
                        }
                        catch (ArgumentException)
                        {
                            m_Encoding = CrawlOrder.PrimEncoding;
                        }
                    }
                }
            }
        }

        /*
         * Receive the message body
         */
        public String GetContentString()
        {
            if (!m_Socket.Connected)
                return String.Empty;

            /*
             * if the status is not s_succeed we still try to download the message body, such as
             * some error the server still sent the message to client.
             */

            //spider status changed...
            m_Controller.Xbuject.OnSpiderStatusChanged(new SpiderStatusPackage(Int32.Parse(Thread.CurrentThread.Name), 
                                                                               CrawlSpider.SpiderStatus.Downloading
                                                                               )
                                                      );
            /*
             * this value is used when the receive method throwes out 10035(WouldBlock) exception
             * when this error occurs, just retry another 's_ReTryTimes' times to receive the data
             * if still not gets it...give up...
             */
            Int32 flagforError = m_Controller.m_CrawlOrder.m_BlockedRetry;
            /*
             * check if the data was transfer by chunk
             */
            if (m_Headers["transfer-encoding"] != null &&
                m_Headers["transfer-encoding"].ToLower() == "chunked")
            {
                //chunked-data, ignore the content-length header part...
                m_ContentLength = -1;
                //get the chunked date
                return GetChunkedContent();
            }

            Int32  nBytes      = 0;
            Int32  nTotalBytes = 0;
            String strResponse = String.Empty;

            Byte[] ReceiveBuffer = new Byte[s_ContentBufferSize];

            try
            {
                /*
                 * if got content-length, to receive the data with the exact length: nSize(m_ContentLength)...
                 * if nSize == -1 means that the server do not tell the exact length of message-body
                 */
                if (m_ContentLength >= 0)
                {
                    strResponse += ReceiveEx(ReceiveBuffer, 0, m_ContentLength, SocketFlags.None);
                }
                //otherwise...try to receive all the available data...
                else
                {
                    while (m_Socket.Connected && (nBytes = m_Socket.Receive(ReceiveBuffer, 0, s_ContentBufferSize, SocketFlags.None)) > 0)
                    {
                        nTotalBytes += nBytes;
                        strResponse += m_Encoding.GetString(ReceiveBuffer, 0, nBytes);

                        //trigger the downloading event
                        m_Controller.Xbuject.OnDownLoading(new DownLoadingPackage(Int32.Parse(Thread.CurrentThread.Name), 
                                                                                  nTotalBytes/*means the bytes received*/, 
                                                                                  0
                                                                                  )
                                                          );

                        Thread.Sleep(50);
                    }

                    m_Headers["content-length"] = nTotalBytes.ToString();
                }
            }
            catch (SocketException se)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_ReceiveData;
                m_Uri.m_SocketError = se.Message;
            }
            catch (ArgumentException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_ReceiveData;
            }
            catch (ObjectDisposedException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_ReceiveData;
            }
            catch (NotReceivedExpectedSizeException)
            {
                m_Uri.m_Status = CrawlUri.StatusCode.E_NeedRetry;
            }

            m_Controller.Xbuject.OnSpiderStatusChanged(new SpiderStatusPackage(Int32.Parse(Thread.CurrentThread.Name), 
                                                                               CrawlSpider.SpiderStatus.Downloaded
                                                                               )
                                                      );
            return strResponse;
        }

        /*
         * the chunked data's structure goes like this:
         * 
         * '[Chunk-size][\r\n][Chunked-data][\r\n][Chunk-size][\r\n][Chunked-data][\r\n][0][\r\n]'
         * 
         * when the chunk-size equals zero, flags the data has transfered completely.
         */
        protected String GetChunkedContent()
        {
            /*
             * In RFC2068 chunked data defined as follows...
             * 
             * Chunked-Body   = *chunk
             *                  "0" CRLF
             *                  footer
             *                  CRLF
             *
             * chunk          = chunk-size [ chunk-ext ] CRLF
             *                  chunk-data CRLF
             *
             * hex-no-zero    = <HEX excluding "0">
             *
             * chunk-size     = hex-no-zero *HEX
             * chunk-ext      = *( ";" chunk-ext-name [ "=" chunk-ext-value ] )
             * chunk-ext-name = token
             * chunk-ext-val  = token | quoted-string
             * chunk-data     = chunk-size(OCTET)
             *
             * footer         = *entity-header
             *
             */
            Int32 flagforError = m_Controller.m_CrawlOrder.m_BlockedRetry;

            Int32 nSize       = 0;
            Int32 nTotalBytes = 0;

            String strtemp;
            String strChunksize;
            String strResponse = String.Empty;

            //flags if the chunk meets its end
            Boolean bflag;
            //flags if has chunk-ext in chunk bodys...
            Boolean bsemicolon;

            Byte[] chunk      = new Byte[s_HeaderBufferSize];
            Byte[] chunk_data = new Byte[s_ContentBufferSize];

            while (true)
            {
                //reset the flags and chunk size
                bflag = false; bsemicolon = false; strChunksize = "";

                try
                {
                    //try to receive the chunk size to the buffer...
                    while (true)
                    {
                        strtemp = ReceiveEx(chunk, 0, 1, SocketFlags.None);

                        if (bflag) break;
                        if (strtemp == "\r")
                            bflag = true;
                        else if (strtemp == ";")
                            bsemicolon = true;
                        //add byte to the strChunksize before the semicolon...
                        else if(!bsemicolon)
                            strChunksize += strtemp;
                    }
                    //parsing the chunk size...
                    nSize = Int32.Parse(strChunksize, System.Globalization.NumberStyles.HexNumber);

                    //has met the end...
                    if (nSize == 0)
                    {
                        //received the '\r\n' at the tail of the chunked-data
                        ReceiveEx(chunk, 0, 2, SocketFlags.None);
                        break;
                    }

                    //try to receive the chunked-data by the specified size
                    strResponse += ReceiveEx(chunk_data, 0, nSize, SocketFlags.None);
                    //force to read the '/r/n'...
                    ReceiveEx(chunk, 0, 2, SocketFlags.None);
                }
                catch (SocketException se)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_ReceiveChunkedData;
                    m_Uri.m_SocketError = se.Message;
                    break;
                }
                catch (ArgumentException)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_ReceiveChunkedData;
                    break;
                }
                catch (ObjectDisposedException)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_ReceiveChunkedData;
                    break;
                }
                catch (NotReceivedExpectedSizeException)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_NeedRetry;
                    break;
                }
            }

            if (m_Uri.m_Status == CrawlUri.StatusCode.S_Succeed)
            {
                /*
                 * receive the footer (if has) of the chunck...
                 */
                try
                {
                    bflag = false;
                    while (m_Socket.Available > 0)
                    {
                        ReceiveEx(chunk, 0, 1, SocketFlags.None);
                        {
                            if (bflag) break;
                            if (chunk[0] == '\r')
                                bflag = true;
                        }
                    }

                    m_Headers["content-length"] = nTotalBytes.ToString();
                    m_Headers.Remove("transfer-encoding");
                }
                catch (SocketException se)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_ReceiveChunkedData;
                    m_Uri.m_SocketError = se.Message;
                }
                catch (ArgumentException)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_ReceiveChunkedData;
                }
                catch (ObjectDisposedException)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_ReceiveChunkedData;
                }
                catch (NotReceivedExpectedSizeException)
                {
                    m_Uri.m_Status = CrawlUri.StatusCode.E_NeedRetry;
                }
            }

            m_Controller.Xbuject.OnSpiderStatusChanged(new SpiderStatusPackage(Int32.Parse(Thread.CurrentThread.Name), 
                                                                               CrawlSpider.SpiderStatus.Downloaded
                                                                               )
                                                      );

            return strResponse;
        }

        public void Close() 
        {
            m_KeepAlive = false;

            if(m_Socket.Connected)
                m_Socket.Shutdown(SocketShutdown.Both);
            m_Socket.Close(); 
        }

        /*
         * behaviour like the Socket.Receive function
         * just retry some times receive the data when the TCP/IP stack blocking...
         * 
         * Notice: this function Force to receive "nSize" of data...or... timeout...
         */
        protected String ReceiveEx(Byte[] buffer, Int32 offset, Int32 nSize, SocketFlags socketFlags)
        {
            Int32 TempToRecieved = nSize;
            Int32 flagforError   = m_Controller.m_CrawlOrder.m_BlockedRetry;
            /*
             * if Receive return 0 for m_TimeOut seconds
             * teminate the function and throw an exception.
             */
            Int32 nTry   = (m_TimeOut/1000/*ms*/) * 20;
            Int32 nBytes = 0;
            /*
             * deal with the high frequency processing of String
             */
            StringBuilder res = new StringBuilder(String.Empty);

            while(m_Socket.Connected && nSize > 0 && nTry > 0)
            {
                try
                {
                    if (nSize > buffer.Length - offset)
                        nBytes = m_Socket.Receive(buffer, offset, buffer.Length - offset, socketFlags);
                    else
                        nBytes = m_Socket.Receive(buffer, offset, nSize, socketFlags);

                    if (nBytes > 0)
                    {
                        nSize -= nBytes;
                        res.Append(m_Encoding.GetString(buffer, offset, nBytes));
                    }
                    else
                    {
                        Thread.Sleep(100);
                        nTry--;
                    }
                }
                catch (SocketException se)
                {
                    if ( se.SocketErrorCode == SocketError.WouldBlock ||
                         se.SocketErrorCode == SocketError.IOPending  ||
                         se.SocketErrorCode == SocketError.NoBufferSpaceAvailable
                       )
                    {
                        if (flagforError > 0)
                        {
                            /*
                             * just sleep sometimes...
                             * then retry to fetch the data again...
                             */
                            Thread.Sleep(500);
                            flagforError--;
                        }
                        else
                            throw se;
                    }
                    else
                        throw se;
                }

                /*
                 * trigger the downloading event
                 * Notice: when the TempToRecieved smaller than 2 that means receive the Http-header or
                 *         RC-FL which we do not care...
                 */
                if (TempToRecieved > 2)
                    m_Controller.Xbuject.OnDownLoading(new DownLoadingPackage(Int32.Parse(Thread.CurrentThread.Name), 
                                                                              res.Length,
                                                                              m_ContentLength
                                                                              )
                                                      );
            }

            if (nTry == 0)
                throw new NotReceivedExpectedSizeException("Not Receive the expected size of data...");
            if (!m_Socket.Connected)
                throw new SocketException(10057); //SocketError.NotConnected;

            return res.ToString();
        }

        protected Int32 SendEx(Byte[] buffer)
        {
            Int32 flagforError = m_Controller.m_CrawlOrder.m_BlockedRetry;
            Int32 nBytes       = 0;
            Int32 nTotalBytes  = buffer.Length;

            while (nBytes < nTotalBytes)
            {
                try
                {
                    nBytes += m_Socket.Send(buffer);
                    //reach here means the data send successfully...
                }
                catch (SocketException se)
                {
                    if ( se.SocketErrorCode == SocketError.WouldBlock ||
                         se.SocketErrorCode == SocketError.IOPending  ||
                         se.SocketErrorCode == SocketError.NoBufferSpaceAvailable
                       )
                    {
                        if (flagforError > 0)
                        {
                            /*
                             * just sleep sometimes...
                             * then retry to fetch the data again...
                             */
                            Thread.Sleep(500);
                            flagforError--;
                        }
                        else
                            throw se;
                    }
                    else
                        throw se;
                }
            }
            return nTotalBytes;
        }

        public class NotReceivedExpectedSizeException : Exception
        {
            public NotReceivedExpectedSizeException()
                :base() { }
            public NotReceivedExpectedSizeException(String msg)
                : base(msg) { }
        }

        private WebHeaderCollection m_Headers;
        private CrawlUri m_Uri;

        private String   m_RedirectedPage;
        private String   m_Header;

        private Encoding m_Encoding;

        //Notice: m_ContentLength == -1 means chunked.
        private Int32 m_ContentLength;
        private Int32 m_TimeOut;
        internal CrawlController m_Controller;

        internal Socket  m_Socket;
        internal Boolean m_KeepAlive;
    }
}
